package producers;

import java.util.ArrayList;
import confInfo.*;
import codeComposition.*;
import util.*;

public class SpiderProducer implements Producer {
	ModelElement sme;
	ConfBean confBean;
	ArrayList<ArrayList<ItemBean>> levelsItems;
	String name;
	public SpiderProducer(String name, ConfBean confBean){
		this.name = name;
		this.confBean = confBean;
		this.levelsItems = confBean.getLevelsItems();
		sme = new ModelElement(name);
		this.addImport();
		
		FunctionElement blank_filter = formBlankFilterFunc();
		this.addItemLoader(blank_filter);
		
		sme.addExpression(blank_filter);
		ClassElement spiderClass = formSpiderClass(name, confBean);
//		sme.addExpression(formCallBackFunction("func","\"======xpath======\"", "self.nextfunc"));
		sme.addClass(spiderClass);
	}	
	
	public void addImport(){
		sme.addImport(new ImportElement("scrapy.contrib_exp.crawlspider", "CrawlSpider, Rule"));
		sme.addImport(new ImportElement("scrapy.http","Request"));
		sme.addImport(new ImportElement("scrapy.selector","HtmlXPathSelector"));
		sme.addImport(new ImportElement("scrapy.http","FormRequest"));
		sme.addImport(new ImportElement("scrapy.contrib.loader","XPathItemLoader"));
		sme.addImport(new ImportElement("scrapy.contrib.loader.processor","Compose"));
		sme.addImport(new ImportElement("re"));
		sme.addImport(new ImportElement(this.name+".items",StringHelper.firtCharToUpperCase(this.name)+"Item"));
	}
	public void addItemLoader(FunctionElement blank_filter){
		ClassElement ce = new ClassElement(this.name + "ItemLoader", "XPathItemLoader");
		
		ArrayList<Element> argus = new ArrayList<Element>();
		argus.add(new ReferenceElement(blank_filter.funcName));
		ConstructElement construct = new ConstructElement("Compose", 
				argus, "default_output_processor");
		ce.add(construct);
		sme.addClass(ce);
	}
	FunctionElement formBlankFilterFunc(){
		FunctionElement f = new FunctionElement("blankFilter", "text", 0);
		ArrayList<Element> argus = new ArrayList<Element>();
		argus.add(new ReferenceElement("u'\\xa0'"));
		argus.add(new ReferenceElement("''"));
		f.add(new CallFuctionElement("text[0]", "replace", argus, "text", 1));
		
		ArrayList<Element> argus2 = new ArrayList<Element>();
		argus2.add(new ReferenceElement("'\\s'"));
		argus2.add(new ReferenceElement("''"));		
		argus2.add(new ReferenceElement("text"));
		f.add(new CallFuctionElement("re", "sub", argus2, "text", 1));
		f.add(new ReturnElement("text", 1));
		
		return f;
	}
	FunctionElement formCallBackFunction(String func, String xpath, String nextFunc){
		FunctionElement f = new FunctionElement(func, Translator.stringsToElements("self","response"), 1);
		ConstructElement ce = new ConstructElement("HtmlXPathSelector", Translator.stringsToElements("response"), "hxs",1);
		f.add(ce);
		f.add(new CallFuctionElement("hxs", "select", Translator.stringsToElements(xpath), "result", 1));
		SimpleExpressionElement see = new SimpleExpressionElement(2, "if len(result.extract()) == 0:", "\treturn");
		f.add(see);
		
		CallFuctionElement cfe = new CallFuctionElement("result","extract");
		ListItemElement argu1 = new ListItemElement(cfe, String.valueOf(0));
		VarAssignElement argu2 = new VarAssignElement(nextFunc,"callback");
		ce = new ConstructElement("FormRequest", Translator.stringsToElements(argu1.toString(),argu2.getRawString()));
		ReturnElement re = new ReturnElement(ce.getRawString(), 1);
		f.add(re);
		
		return f;
	}
	
	FunctionElement formGetItemsFunction(String name, ArrayList<ItemBean> fields){
		FunctionElement f = new FunctionElement(name, Translator.stringsToElements("self","response"), 1);
		ConstructElement ce1 = new ConstructElement("HtmlXPathSelector", Translator.stringsToElements("response"), "hxs",1);
		ConstructElement ce2 = new ConstructElement(StringHelper.firtCharToUpperCase(this.name)+"Item",
				"item",1);
		ListItemElement li = new ListItemElement("item","'url'");
		VarAssignElement va1 = new VarAssignElement("response.url",li.toString());
		VarAssignElement va2 = new VarAssignElement(StringHelper.firtCharToUpperCase(this.name)+
				"ItemLoader(item=item, selector=hxs)","l");
		f.add(ce1);
		f.add(ce2);
		f.add(va1);
		f.add(va2);
		
		for(ItemBean ib:fields){
			addXpath(f, ib.getKey(),ib.getValue());
		}
		ReturnElement re = new ReturnElement("l.load_item()", 1);
		f.add(re);
		return f;
	}
	public void addXpath(FunctionElement f, String key, String value){
		if(key.trim() == ""|| value.trim()=="") return;
		CallFuctionElement cfe = new CallFuctionElement(
				"l","add_xpath", Translator.stringsToElements("'"+key+"'", "'"+value+"'"));
		f.add(cfe);
	}
	
	ClassElement formSpiderClass(String name, ConfBean confBean){
		ClassElement spiderClass = new ClassElement(name+"Spider", "CrawlSpider");
		VarAssignElement ve = new VarAssignElement("'"+name+"'", "name");
		spiderClass.add(ve);
		spiderClass.add(new VarAssignElement("['"+confBean.getDomain()+"']", "allowed_domains")); 
		
		//add start_urls
		ArrayList<ItemBean> startUrls = confBean.getLevelsItems().get(0);
		ArrayList<String> s = new ArrayList<String>();
		for(ItemBean ib:startUrls){
			if(ib.getValue().trim() == "") continue;
			s.add("'" + ib.getValue().trim() + "'");
		}
		ListAssighElement lae = new ListAssighElement(s, "start_urls");
		spiderClass.add(lae);
		
		//add rules
		ArrayList<ItemBean> rules = confBean.getLevelsItems().get(1);
		SimpleExpressionElement see = new SimpleExpressionElement();
		see.add("rules = (");
		for(ItemBean ib:rules){
			if(ib.getKey().trim() == "") continue;
			ConstructElement ce = new ConstructElement("Rule", 
					Translator.stringsToElements("r'"+ib.getValue().trim()+"'","'"+ib.getKey().trim()+"Func'"));
			see.add("\t"+ce.getRawString()+",");
		}
		see.add(")");
		spiderClass.add(see);
		
		addCallbackFuncToSpider(spiderClass, confBean);
		spiderClass.add(this.formGetItemsFunction("itemFunc", levelsItems.get(levelsItems.size()-1)));
		return spiderClass;
	}
	
	private void addCallbackFuncToSpider(ClassElement spiderClass, ConfBean confBean){
		for(ArrayList<ItemBean> iblist:confBean.getLevelsItems()){
			for(ItemBean ib:iblist){
				if(ib.getLevel()=="rule"||ib.getLevel()=="start_url"||ib.getLevel()=="item") 
					continue;
				FunctionElement f = formCallBackFunction(
						ib.getLevel()+"Func","'" + ib.getValue()+"'", "self."+ib.getKey().trim()+"Func");
				spiderClass.add(f);
			}
		}
	}
	public String produce() {
		// TODO Auto-generated method stub
		return sme.toString();
	}
}
